{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Credential Extraction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FGdump"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-19T06:42:38.534353Z",
     "iopub.status.busy": "2020-10-19T06:42:38.534070Z",
     "iopub.status.idle": "2020-10-19T06:42:42.611080Z",
     "shell.execute_reply": "2020-10-19T06:42:42.610567Z",
     "shell.execute_reply.started": "2020-10-19T06:42:38.534328Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "87f0ac14dd7e4449bc0f97afcbcaee94",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=4.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Finished.                     "
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start_time</th>\n",
       "      <th>cmd_line</th>\n",
       "      <th>entities</th>\n",
       "      <th>process_name</th>\n",
       "      <th>parent_process_name</th>\n",
       "      <th>end_time</th>\n",
       "      <th>process_path</th>\n",
       "      <th>input_event</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-09-14 20:57:28</td>\n",
       "      <td>-v</td>\n",
       "      <td>[eyJlbnRpdHlUeXBlIjoiVVNFUiIsInByaW1hcnlBcnRpZ...</td>\n",
       "      <td>cachedump64.exe</td>\n",
       "      <td>C:\\Users\\Administrator\\Downloads\\fgdump-2.1.0-...</td>\n",
       "      <td>2020-09-14 20:57:28</td>\n",
       "      <td>c:\\users\\admini~1\\appdata\\local\\temp</td>\n",
       "      <td>{'_tenant': 'test', '_time': '1600117048000', ...</td>\n",
       "      <td>2020-09-14 20:57:28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           start_time  ...           timestamp\n",
       "0 2020-09-14 20:57:28  ... 2020-09-14 20:57:28\n",
       "\n",
       "[1 rows x 9 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<spl2_kernel.spl2_runner.SPL2Job at 0x7f11b3a171d0>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%spl2 --parallelism=7\n",
    "| from read_text(\"s3://smle-experiments/datasets/ssa/misko_ssa_detections/logFgdump.log\")\n",
    "| select from_json_object(value) as input_event\n",
    "| eval timestamp=parse_long(ucast(map_get(input_event, \"_time\"), \"string\", null)),\n",
    "       cmd_line=ucast(map_get(input_event, \"process\"), \"string\", null),\n",
    "       process_name=ucast(map_get(input_event, \"process_name\"), \"string\", null),\n",
    "       process_path=ucast(map_get(input_event, \"process_path\"), \"string\", null),\n",
    "       parent_process_name=ucast(map_get(input_event, \"parent_process_name\"), \"string\", null)\n",
    "| where cmd_line != null AND\n",
    "        match_regex(parent_process_name, /(?i)fgdump.exe/)=true\n",
    "| eval start_time = timestamp,\n",
    "       end_time = timestamp,\n",
    "       entities = mvappend( ucast(map_get(input_event, \"dest_user_id\"), \"string\", null), \n",
    "                            ucast(map_get(input_event, \"dest_device_id\"), \"string\", null));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-19T07:31:33.752874Z",
     "iopub.status.busy": "2020-10-19T07:31:33.752605Z",
     "iopub.status.idle": "2020-10-19T07:31:36.961722Z",
     "shell.execute_reply": "2020-10-19T07:31:36.960969Z",
     "shell.execute_reply.started": "2020-10-19T07:31:33.752852Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ed2f12e58284468be940ca472904ca6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Finished.                     "
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start_time</th>\n",
       "      <th>cmd_line</th>\n",
       "      <th>entities</th>\n",
       "      <th>process_name</th>\n",
       "      <th>parent_process_name</th>\n",
       "      <th>end_time</th>\n",
       "      <th>process_path</th>\n",
       "      <th>input_event</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-09-14 20:57:28</td>\n",
       "      <td>\"\"C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\cachedu...</td>\n",
       "      <td>[eyJlbnRpdHlUeXBlIjoiVVNFUiIsInByaW1hcnlBcnRpZ...</td>\n",
       "      <td>cachedump64.exe</td>\n",
       "      <td>C:\\Windows\\System32\\services.exe</td>\n",
       "      <td>2020-09-14 20:57:28</td>\n",
       "      <td>c:\\users\\admini~1\\appdata\\local\\temp</td>\n",
       "      <td>{'_tenant': 'test', '_time': '1600117048000', ...</td>\n",
       "      <td>2020-09-14 20:57:28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           start_time  ...           timestamp\n",
       "0 2020-09-14 20:57:28  ... 2020-09-14 20:57:28\n",
       "\n",
       "[1 rows x 9 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<spl2_kernel.spl2_runner.SPL2Job at 0x7f1180aec090>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%spl2 --parallelism=7\n",
    "| from read_text(\"s3://smle-experiments/datasets/ssa/misko_ssa_detections/logFgdump.log\")\n",
    "| select from_json_object(value) as input_event\n",
    "| eval timestamp=parse_long(ucast(map_get(input_event, \"_time\"), \"string\", null)),\n",
    "       cmd_line=ucast(map_get(input_event, \"process\"), \"string\", null),\n",
    "       process_name=ucast(map_get(input_event, \"process_name\"), \"string\", null),\n",
    "       process_path=ucast(map_get(input_event, \"process_path\"), \"string\", null),\n",
    "       parent_process_name=ucast(map_get(input_event, \"parent_process_name\"), \"string\", null)\n",
    "| where cmd_line != null AND process_name != null AND parent_process_name != null AND\n",
    "        match_regex(parent_process_name, /(?i)System32\\\\services.exe/)=true AND\n",
    "        match_regex(process_name, /(?i)cachedump\\d{0,2}.exe/)=true AND\n",
    "        match_regex(process_path, /(?i)\\\\Temp/)=true AND\n",
    "        match_regex(cmd_line, /(?i)\\-s/)=true \n",
    "| eval start_time = timestamp,\n",
    "       end_time = timestamp,\n",
    "       entities = mvappend( ucast(map_get(input_event, \"dest_user_id\"), \"string\", null), \n",
    "                            ucast(map_get(input_event, \"dest_device_id\"), \"string\", null));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-19T06:43:38.657607Z",
     "iopub.status.busy": "2020-10-19T06:43:38.657338Z",
     "iopub.status.idle": "2020-10-19T06:43:43.943417Z",
     "shell.execute_reply": "2020-10-19T06:43:43.942731Z",
     "shell.execute_reply.started": "2020-10-19T06:43:38.657582Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f54e0d67a72f4885b5728174656682dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Finished.                     "
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start_time</th>\n",
       "      <th>cmd_line</th>\n",
       "      <th>entities</th>\n",
       "      <th>process_name</th>\n",
       "      <th>end_time</th>\n",
       "      <th>process_path</th>\n",
       "      <th>input_event</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-09-14 20:57:28</td>\n",
       "      <td>-v</td>\n",
       "      <td>[eyJlbnRpdHlUeXBlIjoiVVNFUiIsInByaW1hcnlBcnRpZ...</td>\n",
       "      <td>cachedump64.exe</td>\n",
       "      <td>2020-09-14 20:57:28</td>\n",
       "      <td>c:\\users\\admini~1\\appdata\\local\\temp</td>\n",
       "      <td>{'_tenant': 'test', '_time': '1600117048000', ...</td>\n",
       "      <td>2020-09-14 20:57:28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           start_time  ...           timestamp\n",
       "0 2020-09-14 20:57:28  ... 2020-09-14 20:57:28\n",
       "\n",
       "[1 rows x 8 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<spl2_kernel.spl2_runner.SPL2Job at 0x7f1180b99510>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%spl2 --parallelism=7\n",
    "| from read_text(\"s3://smle-experiments/datasets/ssa/misko_ssa_detections/logFgdump.log\")\n",
    "| select from_json_object(value) as input_event\n",
    "| eval timestamp=parse_long(ucast(map_get(input_event, \"_time\"), \"string\", null)),\n",
    "       cmd_line=ucast(map_get(input_event, \"process\"), \"string\", null),\n",
    "       process_name=ucast(map_get(input_event, \"process_name\"), \"string\", null),\n",
    "       process_path=ucast(map_get(input_event, \"process_path\"), \"string\", null)\n",
    "| where cmd_line != null AND process_name != null AND process_path != null AND\n",
    "        match_regex(process_name, /(?i)cachedump\\d{0,2}.exe/)=true AND\n",
    "        match_regex(process_path, /(?i)\\\\Temp/)=true AND\n",
    "        match_regex(cmd_line, /(?i)\\-v/)=true \n",
    "| eval start_time = timestamp,\n",
    "       end_time = timestamp,\n",
    "       entities = mvappend( ucast(map_get(input_event, \"dest_user_id\"), \"string\", null), \n",
    "                            ucast(map_get(input_event, \"dest_device_id\"), \"string\", null));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LaZagne"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-16T11:11:37.602675Z",
     "iopub.status.busy": "2020-10-16T11:11:37.602388Z",
     "iopub.status.idle": "2020-10-16T11:11:38.897829Z",
     "shell.execute_reply": "2020-10-16T11:11:38.897200Z",
     "shell.execute_reply.started": "2020-10-16T11:11:37.602651Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ea6d405d5dbb45fbbdcccd118a5a74fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Finished.                     "
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>end_time</th>\n",
       "      <th>start_time</th>\n",
       "      <th>entities</th>\n",
       "      <th>process_name</th>\n",
       "      <th>input_event</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-09-12 01:25:41</td>\n",
       "      <td>2020-09-12 01:25:41</td>\n",
       "      <td>[eyJlbnRpdHlUeXBlIjoiVVNFUiIsInByaW1hcnlBcnRpZ...</td>\n",
       "      <td>lazagne.exe</td>\n",
       "      <td>{'_tenant': 'test', '_time': '1599873941000', ...</td>\n",
       "      <td>2020-09-12 01:25:41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-09-12 01:25:41</td>\n",
       "      <td>2020-09-12 01:25:41</td>\n",
       "      <td>[eyJlbnRpdHlUeXBlIjoiVVNFUiIsInByaW1hcnlBcnRpZ...</td>\n",
       "      <td>lazagne.exe</td>\n",
       "      <td>{'_tenant': 'test', '_time': '1599873941000', ...</td>\n",
       "      <td>2020-09-12 01:25:41</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             end_time  ...           timestamp\n",
       "0 2020-09-12 01:25:41  ... 2020-09-12 01:25:41\n",
       "1 2020-09-12 01:25:41  ... 2020-09-12 01:25:41\n",
       "\n",
       "[2 rows x 6 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<spl2_kernel.spl2_runner.SPL2Job at 0x7fc413a60c50>"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%spl2 --parallelism=7\n",
    "| from read_text(\"s3://smle-experiments/datasets/ssa/misko_ssa_detections/logLazagneCredDump.log\")\n",
    "| select from_json_object(value) as input_event\n",
    "| eval timestamp=parse_long(ucast(map_get(input_event, \"_time\"), \"string\", null)),\n",
    "       process_name=ucast(map_get(input_event, \"process_name\"), \"string\", null)\n",
    "| where process_name != null AND\n",
    "        match_regex(process_name, /(?i)lazagne.exe/)=true  \n",
    "| eval start_time = timestamp,\n",
    "       end_time = timestamp,\n",
    "       entities = mvappend( ucast(map_get(input_event, \"dest_user_id\"), \"string\", null), \n",
    "                            ucast(map_get(input_event, \"dest_device_id\"), \"string\", null));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-19T06:44:15.842408Z",
     "iopub.status.busy": "2020-10-19T06:44:15.842154Z",
     "iopub.status.idle": "2020-10-19T06:44:17.835759Z",
     "shell.execute_reply": "2020-10-19T06:44:17.835188Z",
     "shell.execute_reply.started": "2020-10-19T06:44:15.842387Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d20cc7f7621b4e449419b4216187807c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Finished.                     "
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>end_time</th>\n",
       "      <th>start_time</th>\n",
       "      <th>cmd_line</th>\n",
       "      <th>entities</th>\n",
       "      <th>input_event</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-09-12 01:25:41</td>\n",
       "      <td>2020-09-12 01:25:41</td>\n",
       "      <td>lazagne  all -oA -output lazDump.txt</td>\n",
       "      <td>[eyJlbnRpdHlUeXBlIjoiVVNFUiIsInByaW1hcnlBcnRpZ...</td>\n",
       "      <td>{'_tenant': 'test', '_time': '1599873941000', ...</td>\n",
       "      <td>2020-09-12 01:25:41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-09-12 01:25:41</td>\n",
       "      <td>2020-09-12 01:25:41</td>\n",
       "      <td>lazagne  all -oA -output lazDump.txt</td>\n",
       "      <td>[eyJlbnRpdHlUeXBlIjoiVVNFUiIsInByaW1hcnlBcnRpZ...</td>\n",
       "      <td>{'_tenant': 'test', '_time': '1599873941000', ...</td>\n",
       "      <td>2020-09-12 01:25:41</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             end_time  ...           timestamp\n",
       "0 2020-09-12 01:25:41  ... 2020-09-12 01:25:41\n",
       "1 2020-09-12 01:25:41  ... 2020-09-12 01:25:41\n",
       "\n",
       "[2 rows x 6 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<spl2_kernel.spl2_runner.SPL2Job at 0x7f1180b42bd0>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%spl2 --parallelism=7\n",
    "| from read_text(\"s3://smle-experiments/datasets/ssa/misko_ssa_detections/logLazagneCredDump.log\")\n",
    "| select from_json_object(value) as input_event\n",
    "| eval timestamp=parse_long(ucast(map_get(input_event, \"_time\"), \"string\", null)),\n",
    "       cmd_line=ucast(map_get(input_event, \"process\"), \"string\", null)\n",
    "| where cmd_line != null AND\n",
    "        match_regex(cmd_line, /(?i)all\\s+\\-oA\\s+\\-output/)=true  \n",
    "| eval start_time = timestamp,\n",
    "       end_time = timestamp,\n",
    "       entities = mvappend( ucast(map_get(input_event, \"dest_user_id\"), \"string\", null), \n",
    "                            ucast(map_get(input_event, \"dest_device_id\"), \"string\", null));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PowerSploit/DSInternals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-19T06:44:32.330847Z",
     "iopub.status.busy": "2020-10-19T06:44:32.330579Z",
     "iopub.status.idle": "2020-10-19T06:44:34.317758Z",
     "shell.execute_reply": "2020-10-19T06:44:34.317168Z",
     "shell.execute_reply.started": "2020-10-19T06:44:32.330825Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55e657aa20074cbaa58dfbcc45ca5ec0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Finished.                     "
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>end_time</th>\n",
       "      <th>start_time</th>\n",
       "      <th>cmd_line</th>\n",
       "      <th>entities</th>\n",
       "      <th>input_event</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-09-17 18:59:05</td>\n",
       "      <td>2020-09-17 18:59:05</td>\n",
       "      <td>powershell  -command \"\"Get-ADDBAccount -All -D...</td>\n",
       "      <td>[eyJlbnRpdHlUeXBlIjoiVVNFUiIsInByaW1hcnlBcnRpZ...</td>\n",
       "      <td>{'_tenant': 'test', '_time': '1600369145000', ...</td>\n",
       "      <td>2020-09-17 18:59:05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             end_time  ...           timestamp\n",
       "0 2020-09-17 18:59:05  ... 2020-09-17 18:59:05\n",
       "\n",
       "[1 rows x 6 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<spl2_kernel.spl2_runner.SPL2Job at 0x7f1180b36c90>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%spl2 --parallelism=7\n",
    "| from read_text(\"s3://smle-experiments/datasets/ssa/misko_ssa_detections/logPowerShellModule.log\")\n",
    "| select from_json_object(value) as input_event\n",
    "| eval timestamp=parse_long(ucast(map_get(input_event, \"_time\"), \"string\", null)),\n",
    "       cmd_line=ucast(map_get(input_event, \"process\"), \"string\", null)\n",
    "| where cmd_line != null AND\n",
    "        match_regex(cmd_line, /(?i)Get-ADDBAccount/)=true AND\n",
    "        match_regex(cmd_line, /(?i)\\-dbpath[\\s;:\\.\\|]+/)=true\n",
    "| eval start_time = timestamp,\n",
    "       end_time = timestamp,\n",
    "       entities = mvappend( ucast(map_get(input_event, \"dest_user_id\"), \"string\", null), \n",
    "                            ucast(map_get(input_event, \"dest_device_id\"), \"string\", null));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-16T18:40:38.543645Z",
     "iopub.status.busy": "2020-10-16T18:40:38.543331Z",
     "iopub.status.idle": "2020-10-16T18:40:38.546643Z",
     "shell.execute_reply": "2020-10-16T18:40:38.545902Z",
     "shell.execute_reply.started": "2020-10-16T18:40:38.543617Z"
    }
   },
   "source": [
    "### Windows-native debuggers: NTKD, LiveKD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-19T06:44:40.113042Z",
     "iopub.status.busy": "2020-10-19T06:44:40.112773Z",
     "iopub.status.idle": "2020-10-19T06:44:42.191978Z",
     "shell.execute_reply": "2020-10-19T06:44:42.191344Z",
     "shell.execute_reply.started": "2020-10-19T06:44:40.113019Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71f723f491ba4ba29302108a6b3119b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Finished.                     "
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start_time</th>\n",
       "      <th>cmd_line</th>\n",
       "      <th>entities</th>\n",
       "      <th>process_name</th>\n",
       "      <th>end_time</th>\n",
       "      <th>input_event</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-09-14 17:31:20</td>\n",
       "      <td>\"C:\\Program Files (x86)\\Windows Kits\\10\\Debugg...</td>\n",
       "      <td>[eyJlbnRpdHlUeXBlIjoiVVNFUiIsInByaW1hcnlBcnRpZ...</td>\n",
       "      <td>ntkd.exe</td>\n",
       "      <td>2020-09-14 17:31:20</td>\n",
       "      <td>{'_tenant': 'test', '_time': '1600104680000', ...</td>\n",
       "      <td>2020-09-14 17:31:20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           start_time  ...           timestamp\n",
       "0 2020-09-14 17:31:20  ... 2020-09-14 17:31:20\n",
       "\n",
       "[1 rows x 7 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<spl2_kernel.spl2_runner.SPL2Job at 0x7f1180b71d10>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%spl2 --parallelism=7\n",
    "| from read_text(\"s3://smle-experiments/datasets/ssa/misko_ssa_detections/logNtkdDump.log\")\n",
    "| select from_json_object(value) as input_event\n",
    "| eval timestamp=parse_long(ucast(map_get(input_event, \"_time\"), \"string\", null)),\n",
    "       cmd_line=ucast(map_get(input_event, \"process\"), \"string\", null),\n",
    "       process_name=ucast(map_get(input_event, \"process_name\"), \"string\", null)\n",
    "| where cmd_line != null AND process_name != null AND \n",
    "          ( \n",
    "            match_regex(process_name, /^(?i)ntkd\\.exe/)=true OR\n",
    "            match_regex(process_name, /^(?i)kd\\.exe/)=true \n",
    "          ) AND \n",
    "          match_regex(cmd_line, /(?i)\\-z\\s+/)=true\n",
    "| eval start_time = timestamp,\n",
    "       end_time = timestamp,\n",
    "       entities = mvappend( ucast(map_get(input_event, \"dest_user_id\"), \"string\", null), \n",
    "                            ucast(map_get(input_event, \"dest_device_id\"), \"string\", null));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SAME DETECTION ON LiveKD's DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-19T06:44:58.570342Z",
     "iopub.status.busy": "2020-10-19T06:44:58.570056Z",
     "iopub.status.idle": "2020-10-19T06:44:59.753230Z",
     "shell.execute_reply": "2020-10-19T06:44:59.752699Z",
     "shell.execute_reply.started": "2020-10-19T06:44:58.570317Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef72e8a6d72d41b3b8f3c231d0c74e24",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Finished.                     "
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start_time</th>\n",
       "      <th>cmd_line</th>\n",
       "      <th>entities</th>\n",
       "      <th>process_name</th>\n",
       "      <th>end_time</th>\n",
       "      <th>input_event</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-09-12 23:25:47</td>\n",
       "      <td>kd.exe      -z C:\\Windows\\livekd.dmp</td>\n",
       "      <td>[eyJlbnRpdHlUeXBlIjoiVVNFUiIsInByaW1hcnlBcnRpZ...</td>\n",
       "      <td>kd.exe</td>\n",
       "      <td>2020-09-12 23:25:47</td>\n",
       "      <td>{'_tenant': 'test', '_time': '1599953147000', ...</td>\n",
       "      <td>2020-09-12 23:25:47</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           start_time  ...           timestamp\n",
       "0 2020-09-12 23:25:47  ... 2020-09-12 23:25:47\n",
       "\n",
       "[1 rows x 7 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<spl2_kernel.spl2_runner.SPL2Job at 0x7f1180b06710>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%spl2 --parallelism=7\n",
    "| from read_text(\"s3://smle-experiments/datasets/ssa/misko_ssa_detections/logLiveKDFullKernelDump.log\")\n",
    "| select from_json_object(value) as input_event\n",
    "| eval timestamp=parse_long(ucast(map_get(input_event, \"_time\"), \"string\", null)),\n",
    "       cmd_line=ucast(map_get(input_event, \"process\"), \"string\", null),\n",
    "       process_name=ucast(map_get(input_event, \"process_name\"), \"string\", null)\n",
    "| where cmd_line != null AND process_name != null AND \n",
    "          ( \n",
    "            match_regex(process_name, /^(?i)ntkd\\.exe/)=true OR\n",
    "            match_regex(process_name, /^(?i)kd\\.exe/)=true \n",
    "          ) AND \n",
    "          match_regex(cmd_line, /(?i)\\-z\\s+/)=true  \n",
    "| eval start_time = timestamp,\n",
    "       end_time = timestamp,\n",
    "       entities = mvappend( ucast(map_get(input_event, \"dest_user_id\"), \"string\", null), \n",
    "                            ucast(map_get(input_event, \"dest_device_id\"), \"string\", null));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-19T06:45:22.643424Z",
     "iopub.status.busy": "2020-10-19T06:45:22.643152Z",
     "iopub.status.idle": "2020-10-19T06:45:23.764872Z",
     "shell.execute_reply": "2020-10-19T06:45:23.764356Z",
     "shell.execute_reply.started": "2020-10-19T06:45:22.643401Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83fc3507d11841139f30b4ff3d3fcf6c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Finished.                     "
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start_time</th>\n",
       "      <th>cmd_line</th>\n",
       "      <th>entities</th>\n",
       "      <th>process_name</th>\n",
       "      <th>parent_process_name</th>\n",
       "      <th>end_time</th>\n",
       "      <th>input_event</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-09-12 23:25:44</td>\n",
       "      <td>\\??\\C:\\Windows\\system32\\conhost.exe 0xffffffff...</td>\n",
       "      <td>[eyJlbnRpdHlUeXBlIjoiVVNFUiIsInByaW1hcnlBcnRpZ...</td>\n",
       "      <td>conhost.exe</td>\n",
       "      <td>C:\\Program Files (x86)\\Windows Kits\\10\\Debugge...</td>\n",
       "      <td>2020-09-12 23:25:44</td>\n",
       "      <td>{'_tenant': 'test', '_time': '1599953144000', ...</td>\n",
       "      <td>2020-09-12 23:25:44</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           start_time  ...           timestamp\n",
       "0 2020-09-12 23:25:44  ... 2020-09-12 23:25:44\n",
       "\n",
       "[1 rows x 8 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<spl2_kernel.spl2_runner.SPL2Job at 0x7f1180ab5150>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%spl2 --parallelism=7\n",
    "| from read_text(\"s3://smle-experiments/datasets/ssa/misko_ssa_detections/logLiveKDFullKernelDump.log\")\n",
    "| select from_json_object(value) as input_event\n",
    "| eval timestamp=parse_long(ucast(map_get(input_event, \"_time\"), \"string\", null)),\n",
    "       cmd_line=ucast(map_get(input_event, \"process\"), \"string\", null),\n",
    "       process_name=ucast(map_get(input_event, \"process_name\"), \"string\", null),\n",
    "       parent_process_name=ucast(map_get(input_event, \"parent_process_name\"), \"string\", null)\n",
    "| where cmd_line != null AND parent_process_name != null AND process_name != null  AND  \n",
    "          ( match_regex(parent_process_name, /(?i)ntkd\\.exe/)=true OR\n",
    "            match_regex(parent_process_name, /(?i)livekd\\.exe/)=true\n",
    "          ) AND\n",
    "          match_regex(process_name, /(?i)conhost\\.exe/)=true AND\n",
    "          match_regex(cmd_line, /(?i)0xffffffff/)=true AND\n",
    "          match_regex(cmd_line, /(?i)\\-ForceV1/)=true\n",
    "| eval start_time = timestamp,\n",
    "       end_time = timestamp,\n",
    "       entities = mvappend( ucast(map_get(input_event, \"dest_user_id\"), \"string\", null), \n",
    "                            ucast(map_get(input_event, \"dest_device_id\"), \"string\", null));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "SPL2",
   "language": "SPL",
   "name": "spl2"
  },
  "language_info": {
   "mimetype": "text/spl",
   "name": "SPL"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
